package com.shujia.real;

import com.shujia.common.Config;
import com.shujia.sink.ESSink;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction;
import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer;
import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSink;
import org.apache.flink.streaming.connectors.elasticsearch7.RestClientFactory;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.message.BasicHeader;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Requests;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.common.xcontent.XContentType;

import java.util.*;

public class ComputeWeiBoToES {
    public static void main(String[] args) throws Exception {

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();


        Properties kafkaProps = new Properties();
        kafkaProps.setProperty("zookeeper.connect", Config.getString("kafka.zookeeper.connect"));
        kafkaProps.setProperty("bootstrap.servers", Config.getString("kafka.bootstrap.servers"));
        kafkaProps.setProperty("group.id", "asdsadas");

        //构建kafkaSource
        FlinkKafkaConsumer<String> kafkaConsumer = new FlinkKafkaConsumer<>(
                "wz",
                new SimpleStringSchema(),
                kafkaProps);

        kafkaConsumer.setStartFromEarliest();

        SingleOutputStreamOperator<String> filter = env
                .addSource(kafkaConsumer)
                .filter(new FilterFunction<String>() {
                    @Override
                    public boolean filter(String value) throws Exception {
                        return value.split("\\|").length == 8;
                    }
                });


        filter.print();
        filter.addSink(new ESSink());

        env.execute();

    }
}
